package com.libapml.indexer;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;

import com.libapml.util.Arrays;

public class NGramTokenFilter extends TokenFilter {

	private List<Token> bigramBuffer = new ArrayList<Token>();
	private List<Token> trigramBuffer = new ArrayList<Token>();
	
	public NGramTokenFilter(TokenStream input) {
		
		super(input);
	}
	
	private Token buildNgram(List<Token> bufferedTokens) {
		
		String ngramTerm = "";
		for (Token ngramToken : bufferedTokens)
			ngramTerm += new String(Arrays.subarray(
					ngramToken.termBuffer(), ngramToken.termLength())) + " ";
		
		return new Token(
				ngramTerm.trim(), 
				bufferedTokens.get(0).startOffset(),
				bufferedTokens.get(bufferedTokens.size() - 1).endOffset());
	}
	
	public final Token next() throws IOException {

		Token nextToken = null;
		
		if (trigramBuffer.size() == 3) {
			nextToken = buildNgram(trigramBuffer);
			trigramBuffer.remove(0);
		} else {
			if (bigramBuffer.size() == 2) {
				nextToken = buildNgram(bigramBuffer);
				bigramBuffer.remove(0);
			} else {
				if ( (nextToken = input.next()) != null ) {
					trigramBuffer.add(nextToken);
					bigramBuffer.add(nextToken);
				}
			}
		}
		
		return nextToken;
	}

}
